# increase width of Jupyter notebook
from IPython.core.display import display, HTML
display(HTML("<style>.container { width:95% !important; }</style>"))
display(HTML("<style>.output_result { max-width:95% !important; }</style>"))
display(HTML("<style>.prompt { display:none !important; }</style>"))

# download data
website = "https://archive.ics.uci.edu/ml/machine-learning-databases/00286"
folder = "User Identification From Walking Activity"
url = "{}/{}.zip".format(website, folder)
url = url.replace(" ", "%20")
!curl $url -o data.zip

  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 1285k  100 1285k    0     0   565k      0  0:00:02  0:00:02 --:--:--  565k

# unzip data
from zipfile import ZipFile
with ZipFile("data.zip", 'r') as out:
    out.extractall("")

# load data into pandas and visualize all of it
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import math

pd.set_option('display.max_rows', 2500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.max_colwidth', 2000)
pd.set_option('display.width', 2000)
pd.set_option('display.float_format', lambda x: '%.3f' % x)

font_size=18
params = {'legend.fontsize': 10,
          'figure.figsize': (25,5),
          'axes.labelsize': font_size,
          'axes.titlesize': font_size,
          'xtick.labelsize': font_size,
          'ytick.labelsize': font_size,
          'axes.titlepad': 25,
          'figure.max_open_warning': False}
plt.rcParams.update(params)

number_of_users = 22
user_dataframes = {}
users_in_study = []
seconds_per_chart = 40

print("Global visualization of all data:")

for user in range(1,number_of_users+1):
    csv_file = "{}/{}.csv".format(folder, user)
    # publication says that "z_acceleration" should be positive when walking forward, so we switch axes ordering to make that true
    walk_data = pd.read_csv(csv_file, header=None, names=['timestamp', 'x_acceleration', 'z_acceleration', 'y_acceleration'])

    # precision of (x,y,z) accelerometers is high enough such that only extremely rarely should measurements be exactly identical
    # however, several large blocks of data feature an anomaly where data for two axes becomes equal, so we remove this data 
    data_is_not_duplicated = ( (walk_data['x_acceleration'] != walk_data['y_acceleration']) & 
                               (walk_data['y_acceleration'] != walk_data['z_acceleration']) &
                               (walk_data['z_acceleration'] != walk_data['x_acceleration'])
                             )
    walk_data = walk_data[data_is_not_duplicated]
    
    if len(walk_data.index) == 0:
        print("All data for User {} is anomalous (matching sensor values): removing user from study".format(user))
    else:
        user_dataframes[user] = walk_data
        users_in_study.append(user)
    
        # it helps to choose a fixed scale for all data, in order to see its features across users more clearly
        max_timestamp = walk_data['timestamp'].max()
        charts_to_show_for_user = math.ceil(max_timestamp / seconds_per_chart)
        min_timestamp_for_chart = 0

        for chart in range(charts_to_show_for_user):
            max_timestamp_for_chart = min_timestamp_for_chart + seconds_per_chart
            chart_walk_data = walk_data[(walk_data['timestamp'] >= min_timestamp_for_chart) & (walk_data['timestamp'] <= max_timestamp_for_chart)]

            walk_plot = chart_walk_data.plot(  kind='line',
                                               x='timestamp', 
                                               y='x_acceleration', 
                                               color='r', 
                                               legend=True,
                                               title='User {}: {}-{} seconds'.format(user, int(min_timestamp_for_chart), int(max_timestamp_for_chart)),
                                               xlim = [min_timestamp_for_chart, max_timestamp_for_chart],
                                               ylim = [-20.0, 20.0],
                                               yticks=[-20.0, -10.0, 0.0, 10.0, 20.0])

            walk_plot = chart_walk_data.plot(kind='line', x='timestamp', y='y_acceleration', color='g', ax=walk_plot)
            walk_plot = chart_walk_data.plot(kind='line', x='timestamp', y='z_acceleration', color='b', ax=walk_plot)
            walk_plot.set_xlabel('time (s)')
            walk_plot.set_ylabel('acceleration (m/s^2)')

            min_timestamp_for_chart += seconds_per_chart
            max_timestamp_for_chart += seconds_per_chart

Global visualization of all data:
All data for User 19 is anomalous (matching sensor values): removing user from study

# Convert raw time-series data to a sliding window of discrete frames for a LSTM in Keras
# Filter data that is corrupt and noisy

timestamps_per_frame = 100 # approximately 3.0 seconds; hyperparameter that could be learned automatically
number_of_features_in_raw_data = 3 # x_acceleration, y_acceleration, z_acceleration
number_of_features_additionally_requested = 2 # average_acceleration, time_between_observations
number_of_features_per_timestamp = number_of_features_in_raw_data + number_of_features_additionally_requested
percent_training_data_per_user = 0.8 # we split training/testing data proportionally to number of frames per user, before combining

def shuffle_arrays_in_unison(a, b):
    # used for shuffling input data and output labels with the same indexing
    assert len(a) == len(b)
    p = np.random.permutation(len(a))
    return a[p], b[p]

def split_time_series_into_train_test_frames():
    all_train_data = []
    all_train_labels = []
    all_test_data = []
    all_test_labels = []
    for user in users_in_study:
        walk_data = user_dataframes[user]
        number_of_timestamps = len(walk_data.index)
        number_of_frames = math.ceil(number_of_timestamps / timestamps_per_frame) - 1
        frames_for_user = [] 
        min_timestamp_for_frame = 1 # always start on 2nd frame, i.e. frame 1 in frame 0, frame 1, ...
        max_timestamp_for_frame = min_timestamp_for_frame + timestamps_per_frame
        for frame in range(number_of_frames):
            # initialize empty frame, with first dimension of 1 in preparation for stacking there
            frame_data = np.zeros(shape=(1, timestamps_per_frame,number_of_features_per_timestamp), dtype=np.float32)
            frame_walk_data = walk_data.iloc[min_timestamp_for_frame - 1: max_timestamp_for_frame , :]

            # requested feature: time between observations => compute time difference only between each timestamp (available information at run time)
            frame_timestamps = frame_walk_data["timestamp"]
            frame_time_between_observations = frame_timestamps.diff().iloc[1:].values # ignore first NaN value

            # as seen in graphs of data above (e.g. User 18: 0-40 seconds) sometimes past data with smaller timestamps shows up late; we ignore such data
            if not np.all(frame_time_between_observations > 0.0):
                #print("---> User {} data corrupted between timestamp {} - {}".format(user,min_timestamp_for_frame,max_timestamp_for_frame))
                min_timestamp_for_frame += timestamps_per_frame
                max_timestamp_for_frame += timestamps_per_frame
                continue

            # requested feature: average acceleration => average of absolute values of x,y,z for each timestamp
            xyz_acceleration = frame_walk_data[['x_acceleration', 'y_acceleration', 'z_acceleration']]
            average_acceleration = xyz_acceleration.abs().mean(axis=1) # absolute values highlight more vs. less motion
            average_acceleration = average_acceleration.iloc[1:].values # ignore first value from previous frame

            # get raw data
            x_acceleration = frame_walk_data["x_acceleration"].iloc[1:].values 
            y_acceleration = frame_walk_data["y_acceleration"].iloc[1:].values
            z_acceleration = frame_walk_data["z_acceleration"].iloc[1:].values

            frame_data[0,:,0] = x_acceleration
            frame_data[0,:,1] = y_acceleration
            frame_data[0,:,2] = z_acceleration
            frame_data[0,:,3] = average_acceleration
            frame_data[0,:,4] = frame_time_between_observations

            frames_for_user.append(frame_data)

            min_timestamp_for_frame += timestamps_per_frame
            max_timestamp_for_frame += timestamps_per_frame

        frames_for_user = np.vstack(frames_for_user)
        number_of_frames_for_user = frames_for_user.shape[0]
        
        labels_for_user = np.full(shape=(number_of_frames_for_user), fill_value=user-1, dtype=np.uint16) # note: User 1 -> label 0

        # shuffle data so that our training / testing split is not correlated to a specific period of data
        shuffled_frames, shuffled_labels = shuffle_arrays_in_unison(frames_for_user, labels_for_user)
        
        # by splitting training / testing data within each class, we ensure that sufficient training data for every user is encountered
        max_training_frame_index = int(percent_training_data_per_user * number_of_frames_for_user)
        n_train_frames = max_training_frame_index - 1
        n_test_frames = number_of_frames_for_user - n_train_frames
        
        train_data = shuffled_frames[:max_training_frame_index]
        test_data = shuffled_frames[max_training_frame_index:]
        train_labels = shuffled_labels[:max_training_frame_index]
        test_labels = shuffled_labels[max_training_frame_index:]

        print("User {}: {} frames ({} train, {} test), {} timestamps per frame, {} features per timestamp".format(user, number_of_frames_for_user, n_train_frames, n_test_frames, timestamps_per_frame, number_of_features_per_timestamp))

        all_train_data.append(train_data)
        all_test_data.append(test_data)
        all_train_labels.append(train_labels)
        all_test_labels.append(test_labels)       

    all_train_data = np.vstack(all_train_data)
    all_test_data = np.vstack(all_test_data)
        
    all_train_labels = np.concatenate(all_train_labels)# np.expand_dims(, axis=1)
    all_test_labels = np.concatenate(all_test_labels) # np.expand_dims(, axis=1)

    # finally, we shuffle the training and testing data across all users, so that right away our network is training / testing across all users
    train_X, train_Y = shuffle_arrays_in_unison(all_train_data, all_train_labels)
    test_X, test_Y = shuffle_arrays_in_unison(all_test_data, all_test_labels)

    print("\nShape of training data: {}".format(train_X.shape))
    print("Shape of training labels: {}".format(train_Y.shape))
    print("Shape of testing data: {}".format(test_X.shape))
    print("Shape of testing labels: {}".format(test_Y.shape))
    
    return train_X, test_X, train_Y, test_Y

train_X, test_X, train_Y, test_Y = split_time_series_into_train_test_frames()

User 1: 50 frames (39 train, 11 test), 100 timestamps per frame, 5 features per timestamp
User 2: 38 frames (29 train, 9 test), 100 timestamps per frame, 5 features per timestamp
User 3: 11 frames (7 train, 4 test), 100 timestamps per frame, 5 features per timestamp
User 4: 69 frames (54 train, 15 test), 100 timestamps per frame, 5 features per timestamp
User 5: 11 frames (7 train, 4 test), 100 timestamps per frame, 5 features per timestamp
User 6: 49 frames (38 train, 11 test), 100 timestamps per frame, 5 features per timestamp
User 7: 37 frames (28 train, 9 test), 100 timestamps per frame, 5 features per timestamp
User 8: 34 frames (26 train, 8 test), 100 timestamps per frame, 5 features per timestamp
User 9: 79 frames (62 train, 17 test), 100 timestamps per frame, 5 features per timestamp
User 10: 30 frames (23 train, 7 test), 100 timestamps per frame, 5 features per timestamp
User 11: 56 frames (43 train, 13 test), 100 timestamps per frame, 5 features per timestamp
User 12: 47 frames (36 train, 11 test), 100 timestamps per frame, 5 features per timestamp
User 13: 66 frames (51 train, 15 test), 100 timestamps per frame, 5 features per timestamp
User 14: 120 frames (95 train, 25 test), 100 timestamps per frame, 5 features per timestamp
User 15: 36 frames (27 train, 9 test), 100 timestamps per frame, 5 features per timestamp
User 16: 17 frames (12 train, 5 test), 100 timestamps per frame, 5 features per timestamp
User 17: 216 frames (171 train, 45 test), 100 timestamps per frame, 5 features per timestamp
User 18: 187 frames (148 train, 39 test), 100 timestamps per frame, 5 features per timestamp
User 20: 167 frames (132 train, 35 test), 100 timestamps per frame, 5 features per timestamp
User 21: 30 frames (23 train, 7 test), 100 timestamps per frame, 5 features per timestamp
User 22: 96 frames (75 train, 21 test), 100 timestamps per frame, 5 features per timestamp

Shape of training data: (1147, 100, 5)
Shape of training labels: (1147,)
Shape of testing data: (299, 100, 5)
Shape of testing labels: (299,)

from keras.models import Sequential, load_model
from keras.layers import Dense, Flatten, Dropout, LSTM
from keras.utils import to_categorical

# convert integer labels to one-hot encoded vector, so final embedding layer can predict distributions over users
one_hot_train_Y = to_categorical(train_Y) 
one_hot_test_Y = to_categorical(test_Y)

print("Shape of train labels after one-hot encoding: {}".format(one_hot_train_Y.shape))
print("Shape of test labels after one-hot encoding: {}".format(one_hot_test_Y.shape))

# arbitrary LSTM neural network architecture inspired by https://machinelearningmastery.com/how-to-develop-rnn-models-for-human-activity-recognition-time-series-classification/
# for best performance on a real dataset, we could easily automate a search for types of layers, number of layers, hyperparameters of layers, types of activations, ...

number_of_lstm_parameters = 125
probability_of_dropout = 0.25
number_of_dense_parameters = 250
type_of_dense_activation = 'relu'
type_of_loss_function = 'categorical_crossentropy'
type_of_final_activation = 'softmax'
optimizer = 'adam'
metrics = ['accuracy']
input_batch_size = 64 # could be optimized for hardware
epochs_between_tests = 20
number_of_models_to_test = 15

# Define vanilla LSTM architecture
model = Sequential()
model.add(LSTM(number_of_lstm_parameters, input_shape=(timestamps_per_frame, number_of_features_per_timestamp)))
model.add(Dropout(1-probability_of_dropout))
model.add(Dense(number_of_dense_parameters, activation=type_of_dense_activation))
model.add(Dense(number_of_users, activation=type_of_final_activation))
model.compile(loss=type_of_loss_function, optimizer=optimizer, metrics=metrics)
model.summary()

all_test_model_accuracies = np.zeros(number_of_models_to_test, dtype=np.float32)
model_filepaths = []
epochs_trained = []
for testing_index in range(number_of_models_to_test):
    model.fit(train_X, one_hot_train_Y, epochs=epochs_between_tests, batch_size=input_batch_size)

    _, accuracy = model.evaluate(test_X, one_hot_test_Y, batch_size=one_hot_test_Y.shape[0])
    number_of_epochs_trained_for_model = (testing_index+1) * epochs_between_tests
    epochs_trained.append(number_of_epochs_trained_for_model)
    print("Testing accuracy after {} epochs: {:.1f}%\n\n".format(number_of_epochs_trained_for_model, 100*accuracy))
    all_test_model_accuracies[testing_index] = accuracy

    model_filepath = "{}/lstm_walker_recognition_model_{}.h5".format(folder, testing_index)
    model.save(model_filepath)
    model_filepaths.append(model_filepath)

max_model_index = np.argmax(all_test_model_accuracies)
best_model_filepath = model_filepaths[max_model_index]
best_model_test_accuracy = all_test_model_accuracies[max_model_index]
best_model_epochs_trained = epochs_trained[max_model_index]
print("\nThe model trained for {} epochs performed best on held out test data ({:.1f}% accuracy)".format(best_model_epochs_trained, 100*best_model_test_accuracy))
    

Shape of train labels after one-hot encoding: (1147, 22)
Shape of test labels after one-hot encoding: (299, 22)
Model: "sequential_30"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
lstm_29 (LSTM)               (None, 125)               65500     
_________________________________________________________________
dropout_28 (Dropout)         (None, 125)               0         
_________________________________________________________________
dense_56 (Dense)             (None, 250)               31500     
_________________________________________________________________
dense_57 (Dense)             (None, 22)                5522      
=================================================================
Total params: 102,522
Trainable params: 102,522
Non-trainable params: 0
_________________________________________________________________
Epoch 1/20
18/18 [==============================] - 2s 116ms/step - loss: 2.8804 - accuracy: 0.1526
Epoch 2/20
18/18 [==============================] - 2s 122ms/step - loss: 2.5227 - accuracy: 0.2616
Epoch 3/20
18/18 [==============================] - 2s 122ms/step - loss: 2.2776 - accuracy: 0.2877
Epoch 4/20
18/18 [==============================] - 2s 116ms/step - loss: 2.1338 - accuracy: 0.3191
Epoch 5/20
18/18 [==============================] - 2s 120ms/step - loss: 2.0112 - accuracy: 0.3496
Epoch 6/20
18/18 [==============================] - 2s 119ms/step - loss: 2.0286 - accuracy: 0.3235
Epoch 7/20
18/18 [==============================] - 2s 127ms/step - loss: 1.8577 - accuracy: 0.3827
Epoch 8/20
18/18 [==============================] - 2s 116ms/step - loss: 1.7783 - accuracy: 0.3923
Epoch 9/20
18/18 [==============================] - 2s 116ms/step - loss: 1.7102 - accuracy: 0.4037
Epoch 10/20
18/18 [==============================] - 2s 118ms/step - loss: 1.6949 - accuracy: 0.4202
Epoch 11/20
18/18 [==============================] - 2s 119ms/step - loss: 1.7130 - accuracy: 0.4080
Epoch 12/20
18/18 [==============================] - 2s 130ms/step - loss: 1.7422 - accuracy: 0.3967
Epoch 13/20
18/18 [==============================] - 2s 129ms/step - loss: 1.8473 - accuracy: 0.3810
Epoch 14/20
18/18 [==============================] - 2s 127ms/step - loss: 1.7116 - accuracy: 0.4167
Epoch 15/20
18/18 [==============================] - 2s 135ms/step - loss: 1.7087 - accuracy: 0.4080
Epoch 16/20
18/18 [==============================] - 2s 128ms/step - loss: 1.5921 - accuracy: 0.4429
Epoch 17/20
18/18 [==============================] - 2s 115ms/step - loss: 1.5341 - accuracy: 0.4664
Epoch 18/20
18/18 [==============================] - 2s 114ms/step - loss: 1.5178 - accuracy: 0.4647
Epoch 19/20
18/18 [==============================] - 2s 116ms/step - loss: 1.4606 - accuracy: 0.4935
Epoch 20/20
18/18 [==============================] - 2s 117ms/step - loss: 1.5121 - accuracy: 0.4690
1/1 [==============================] - 0s 2ms/step - loss: 1.4340 - accuracy: 0.5385
Testing accuracy after 20 epochs: 53.8%


Epoch 1/20
18/18 [==============================] - 2s 133ms/step - loss: 1.4814 - accuracy: 0.4699
Epoch 2/20
18/18 [==============================] - 2s 123ms/step - loss: 1.4364 - accuracy: 0.5109
Epoch 3/20
18/18 [==============================] - 2s 116ms/step - loss: 1.4365 - accuracy: 0.4969
Epoch 4/20
18/18 [==============================] - 2s 124ms/step - loss: 1.4799 - accuracy: 0.4542
Epoch 5/20
18/18 [==============================] - 2s 120ms/step - loss: 1.4625 - accuracy: 0.4856
Epoch 6/20
18/18 [==============================] - 2s 120ms/step - loss: 1.4466 - accuracy: 0.4795
Epoch 7/20
18/18 [==============================] - 2s 117ms/step - loss: 1.4338 - accuracy: 0.4891
Epoch 8/20
18/18 [==============================] - 2s 118ms/step - loss: 1.4145 - accuracy: 0.4882
Epoch 9/20
18/18 [==============================] - 2s 116ms/step - loss: 1.4198 - accuracy: 0.4978
Epoch 10/20
18/18 [==============================] - 2s 116ms/step - loss: 1.3893 - accuracy: 0.5048
Epoch 11/20
18/18 [==============================] - 2s 116ms/step - loss: 1.3607 - accuracy: 0.5318
Epoch 12/20
18/18 [==============================] - 2s 123ms/step - loss: 1.3256 - accuracy: 0.5336
Epoch 13/20
18/18 [==============================] - 2s 116ms/step - loss: 1.3083 - accuracy: 0.5371
Epoch 14/20
18/18 [==============================] - 2s 116ms/step - loss: 1.3489 - accuracy: 0.5231
Epoch 15/20
18/18 [==============================] - 2s 117ms/step - loss: 1.3359 - accuracy: 0.5458
Epoch 16/20
18/18 [==============================] - 2s 117ms/step - loss: 1.2681 - accuracy: 0.5519
Epoch 17/20
18/18 [==============================] - 2s 117ms/step - loss: 1.2708 - accuracy: 0.5501
Epoch 18/20
18/18 [==============================] - 2s 119ms/step - loss: 1.2269 - accuracy: 0.5876
Epoch 19/20
18/18 [==============================] - 2s 116ms/step - loss: 1.2498 - accuracy: 0.5606
Epoch 20/20
18/18 [==============================] - 2s 117ms/step - loss: 1.3828 - accuracy: 0.5458
1/1 [==============================] - 0s 1ms/step - loss: 1.6178 - accuracy: 0.4649
Testing accuracy after 40 epochs: 46.5%


Epoch 1/20
18/18 [==============================] - 2s 118ms/step - loss: 1.4958 - accuracy: 0.4743
Epoch 2/20
18/18 [==============================] - 2s 117ms/step - loss: 1.3902 - accuracy: 0.4969
Epoch 3/20
18/18 [==============================] - 2s 123ms/step - loss: 1.2799 - accuracy: 0.5484
Epoch 4/20
18/18 [==============================] - 2s 116ms/step - loss: 1.2314 - accuracy: 0.5493
Epoch 5/20
18/18 [==============================] - 2s 115ms/step - loss: 1.2320 - accuracy: 0.5588
Epoch 6/20
18/18 [==============================] - 2s 116ms/step - loss: 1.2302 - accuracy: 0.5545
Epoch 7/20
18/18 [==============================] - 2s 116ms/step - loss: 1.2378 - accuracy: 0.5536
Epoch 8/20
18/18 [==============================] - 2s 117ms/step - loss: 1.2520 - accuracy: 0.5606
Epoch 9/20
18/18 [==============================] - 2s 116ms/step - loss: 1.2027 - accuracy: 0.5693
Epoch 10/20
18/18 [==============================] - 2s 117ms/step - loss: 1.1964 - accuracy: 0.5702
Epoch 11/20
18/18 [==============================] - 2s 117ms/step - loss: 1.1682 - accuracy: 0.5920
Epoch 12/20
18/18 [==============================] - 2s 117ms/step - loss: 1.1492 - accuracy: 0.5920
Epoch 13/20
18/18 [==============================] - 2s 119ms/step - loss: 1.1960 - accuracy: 0.5876
Epoch 14/20
18/18 [==============================] - 2s 116ms/step - loss: 1.1836 - accuracy: 0.5745
Epoch 15/20
18/18 [==============================] - 2s 114ms/step - loss: 1.1256 - accuracy: 0.6120
Epoch 16/20
18/18 [==============================] - 2s 114ms/step - loss: 1.1420 - accuracy: 0.6007
Epoch 17/20
18/18 [==============================] - 2s 115ms/step - loss: 1.1130 - accuracy: 0.6103
Epoch 18/20
18/18 [==============================] - 2s 115ms/step - loss: 1.1448 - accuracy: 0.6251
Epoch 19/20
18/18 [==============================] - 2s 118ms/step - loss: 1.1260 - accuracy: 0.6068
Epoch 20/20
18/18 [==============================] - 2s 118ms/step - loss: 1.0653 - accuracy: 0.6216
1/1 [==============================] - 0s 2ms/step - loss: 1.2213 - accuracy: 0.6187
Testing accuracy after 60 epochs: 61.9%


Epoch 1/20
18/18 [==============================] - 2s 118ms/step - loss: 1.0771 - accuracy: 0.6234
Epoch 2/20
18/18 [==============================] - 2s 117ms/step - loss: 1.1331 - accuracy: 0.5902
Epoch 3/20
18/18 [==============================] - 2s 117ms/step - loss: 1.0763 - accuracy: 0.6286
Epoch 4/20
18/18 [==============================] - 2s 119ms/step - loss: 1.0961 - accuracy: 0.6155
Epoch 5/20
18/18 [==============================] - 2s 116ms/step - loss: 1.1108 - accuracy: 0.6024
Epoch 6/20
18/18 [==============================] - 2s 117ms/step - loss: 1.0310 - accuracy: 0.6399
Epoch 7/20
18/18 [==============================] - 2s 116ms/step - loss: 1.0109 - accuracy: 0.6548
Epoch 8/20
18/18 [==============================] - 2s 114ms/step - loss: 1.1044 - accuracy: 0.6077
Epoch 9/20
18/18 [==============================] - 2s 113ms/step - loss: 1.1281 - accuracy: 0.5946
Epoch 10/20
18/18 [==============================] - 2s 111ms/step - loss: 1.0738 - accuracy: 0.6234
Epoch 11/20
18/18 [==============================] - 2s 111ms/step - loss: 1.0525 - accuracy: 0.6251
Epoch 12/20
18/18 [==============================] - 2s 118ms/step - loss: 1.0372 - accuracy: 0.6443
Epoch 13/20
18/18 [==============================] - 3s 145ms/step - loss: 1.0216 - accuracy: 0.6356
Epoch 14/20
18/18 [==============================] - 2s 119ms/step - loss: 0.9470 - accuracy: 0.6565
Epoch 15/20
18/18 [==============================] - 2s 115ms/step - loss: 0.9538 - accuracy: 0.6565
Epoch 16/20
18/18 [==============================] - 2s 113ms/step - loss: 0.9431 - accuracy: 0.6609
Epoch 17/20
18/18 [==============================] - 2s 115ms/step - loss: 1.0047 - accuracy: 0.6417
Epoch 18/20
18/18 [==============================] - 2s 114ms/step - loss: 0.9713 - accuracy: 0.6495
Epoch 19/20
18/18 [==============================] - 2s 114ms/step - loss: 0.9290 - accuracy: 0.6696
Epoch 20/20
18/18 [==============================] - 2s 113ms/step - loss: 0.9464 - accuracy: 0.6678
1/1 [==============================] - 0s 1ms/step - loss: 1.2648 - accuracy: 0.6154
Testing accuracy after 80 epochs: 61.5%


Epoch 1/20
18/18 [==============================] - 2s 115ms/step - loss: 0.9574 - accuracy: 0.6478
Epoch 2/20
18/18 [==============================] - 2s 114ms/step - loss: 0.9215 - accuracy: 0.6827
Epoch 3/20
18/18 [==============================] - 2s 114ms/step - loss: 0.9637 - accuracy: 0.6504
Epoch 4/20
18/18 [==============================] - 2s 114ms/step - loss: 0.9257 - accuracy: 0.6800
Epoch 5/20
18/18 [==============================] - 2s 112ms/step - loss: 0.9147 - accuracy: 0.6626
Epoch 6/20
18/18 [==============================] - 2s 113ms/step - loss: 0.8815 - accuracy: 0.6975
Epoch 7/20
18/18 [==============================] - 2s 114ms/step - loss: 0.8997 - accuracy: 0.6853
Epoch 8/20
18/18 [==============================] - 2s 112ms/step - loss: 0.9143 - accuracy: 0.6818
Epoch 9/20
18/18 [==============================] - 2s 112ms/step - loss: 0.8464 - accuracy: 0.6949
Epoch 10/20
18/18 [==============================] - 2s 115ms/step - loss: 0.8486 - accuracy: 0.7036
Epoch 11/20
18/18 [==============================] - 2s 113ms/step - loss: 0.7801 - accuracy: 0.7149
Epoch 12/20
18/18 [==============================] - 2s 113ms/step - loss: 0.9377 - accuracy: 0.6696
Epoch 13/20
18/18 [==============================] - 2s 113ms/step - loss: 0.8396 - accuracy: 0.6914
Epoch 14/20
18/18 [==============================] - 2s 117ms/step - loss: 0.7966 - accuracy: 0.7062
Epoch 15/20
18/18 [==============================] - 2s 131ms/step - loss: 0.8088 - accuracy: 0.7114
Epoch 16/20
18/18 [==============================] - 2s 114ms/step - loss: 0.7730 - accuracy: 0.7184
Epoch 17/20
18/18 [==============================] - 2s 114ms/step - loss: 0.7844 - accuracy: 0.7193
Epoch 18/20
18/18 [==============================] - 2s 114ms/step - loss: 0.7918 - accuracy: 0.7201
Epoch 19/20
18/18 [==============================] - 2s 114ms/step - loss: 0.8333 - accuracy: 0.6931
Epoch 20/20
18/18 [==============================] - 2s 114ms/step - loss: 0.8186 - accuracy: 0.7001
1/1 [==============================] - 0s 1ms/step - loss: 1.1761 - accuracy: 0.6455
Testing accuracy after 100 epochs: 64.5%


Epoch 1/20
18/18 [==============================] - 2s 121ms/step - loss: 0.8529 - accuracy: 0.7079
Epoch 2/20
18/18 [==============================] - 2s 112ms/step - loss: 0.8439 - accuracy: 0.7053
Epoch 3/20
18/18 [==============================] - 2s 113ms/step - loss: 0.8658 - accuracy: 0.6879
Epoch 4/20
18/18 [==============================] - 2s 114ms/step - loss: 0.8081 - accuracy: 0.7201
Epoch 5/20
18/18 [==============================] - 2s 115ms/step - loss: 0.8553 - accuracy: 0.7044
Epoch 6/20
18/18 [==============================] - 2s 113ms/step - loss: 0.8519 - accuracy: 0.6940
Epoch 7/20
18/18 [==============================] - 2s 113ms/step - loss: 0.8177 - accuracy: 0.7271
Epoch 8/20
18/18 [==============================] - 2s 116ms/step - loss: 0.7690 - accuracy: 0.7228
Epoch 9/20
18/18 [==============================] - 2s 113ms/step - loss: 0.7694 - accuracy: 0.7201
Epoch 10/20
18/18 [==============================] - 2s 113ms/step - loss: 0.6989 - accuracy: 0.7568
Epoch 11/20
18/18 [==============================] - 2s 113ms/step - loss: 0.7066 - accuracy: 0.7550
Epoch 12/20
18/18 [==============================] - 2s 121ms/step - loss: 0.6713 - accuracy: 0.7681
Epoch 13/20
18/18 [==============================] - 2s 114ms/step - loss: 0.6908 - accuracy: 0.7759
Epoch 14/20
18/18 [==============================] - 2s 112ms/step - loss: 0.6874 - accuracy: 0.7751
Epoch 15/20
18/18 [==============================] - 2s 113ms/step - loss: 0.7797 - accuracy: 0.7428
Epoch 16/20
18/18 [==============================] - 2s 112ms/step - loss: 0.9221 - accuracy: 0.6931
Epoch 17/20
18/18 [==============================] - 2s 114ms/step - loss: 0.8865 - accuracy: 0.7149
Epoch 18/20
18/18 [==============================] - 2s 114ms/step - loss: 0.7842 - accuracy: 0.7524
Epoch 19/20
18/18 [==============================] - 2s 114ms/step - loss: 0.7710 - accuracy: 0.7350
Epoch 20/20
18/18 [==============================] - 2s 113ms/step - loss: 0.7329 - accuracy: 0.7568
1/1 [==============================] - 0s 1ms/step - loss: 1.1685 - accuracy: 0.6555
Testing accuracy after 120 epochs: 65.6%


Epoch 1/20
18/18 [==============================] - 2s 114ms/step - loss: 0.7419 - accuracy: 0.7376
Epoch 2/20
18/18 [==============================] - 2s 113ms/step - loss: 0.7236 - accuracy: 0.7620
Epoch 3/20
18/18 [==============================] - 2s 122ms/step - loss: 0.7457 - accuracy: 0.7446
Epoch 4/20
18/18 [==============================] - 2s 113ms/step - loss: 0.6558 - accuracy: 0.7847
Epoch 5/20
18/18 [==============================] - 2s 114ms/step - loss: 0.6632 - accuracy: 0.7611
Epoch 6/20
18/18 [==============================] - 2s 113ms/step - loss: 0.6546 - accuracy: 0.7751
Epoch 7/20
18/18 [==============================] - 3s 140ms/step - loss: 0.6437 - accuracy: 0.7812
Epoch 8/20
18/18 [==============================] - 2s 121ms/step - loss: 0.6316 - accuracy: 0.7995
Epoch 9/20
18/18 [==============================] - 2s 124ms/step - loss: 0.6159 - accuracy: 0.7916
Epoch 10/20
18/18 [==============================] - 2s 136ms/step - loss: 0.6638 - accuracy: 0.7777
Epoch 11/20
18/18 [==============================] - 3s 148ms/step - loss: 0.6378 - accuracy: 0.7873
Epoch 12/20
18/18 [==============================] - 3s 159ms/step - loss: 0.6927 - accuracy: 0.7620
Epoch 13/20
18/18 [==============================] - 3s 163ms/step - loss: 0.6344 - accuracy: 0.7786
Epoch 14/20
18/18 [==============================] - 3s 140ms/step - loss: 0.5819 - accuracy: 0.8012
Epoch 15/20
18/18 [==============================] - 3s 161ms/step - loss: 0.7403 - accuracy: 0.7768
Epoch 16/20
18/18 [==============================] - 3s 160ms/step - loss: 0.7081 - accuracy: 0.7533
Epoch 17/20
18/18 [==============================] - 3s 164ms/step - loss: 0.7702 - accuracy: 0.7498
Epoch 18/20
18/18 [==============================] - 3s 154ms/step - loss: 0.7069 - accuracy: 0.7777
Epoch 19/20
18/18 [==============================] - 2s 130ms/step - loss: 0.6537 - accuracy: 0.7672
Epoch 20/20
18/18 [==============================] - 3s 154ms/step - loss: 0.6492 - accuracy: 0.7942
1/1 [==============================] - 0s 3ms/step - loss: 1.0967 - accuracy: 0.6957
Testing accuracy after 140 epochs: 69.6%


Epoch 1/20
18/18 [==============================] - 3s 153ms/step - loss: 0.6606 - accuracy: 0.7881
Epoch 2/20
18/18 [==============================] - 2s 116ms/step - loss: 0.6430 - accuracy: 0.7838
Epoch 3/20
18/18 [==============================] - 2s 139ms/step - loss: 0.5583 - accuracy: 0.8152
Epoch 4/20
18/18 [==============================] - 2s 138ms/step - loss: 0.5465 - accuracy: 0.8152
Epoch 5/20
18/18 [==============================] - 3s 145ms/step - loss: 0.5019 - accuracy: 0.8352
Epoch 6/20
18/18 [==============================] - 3s 140ms/step - loss: 0.5166 - accuracy: 0.8335
Epoch 7/20
18/18 [==============================] - 3s 150ms/step - loss: 0.6125 - accuracy: 0.7995
Epoch 8/20
18/18 [==============================] - 2s 114ms/step - loss: 0.6203 - accuracy: 0.7899
Epoch 9/20
18/18 [==============================] - 2s 112ms/step - loss: 0.5901 - accuracy: 0.8073
Epoch 10/20
18/18 [==============================] - 2s 113ms/step - loss: 0.5595 - accuracy: 0.8152
Epoch 11/20
18/18 [==============================] - 2s 118ms/step - loss: 0.6216 - accuracy: 0.8003
Epoch 12/20
18/18 [==============================] - 2s 115ms/step - loss: 0.6198 - accuracy: 0.8003
Epoch 13/20
18/18 [==============================] - 2s 112ms/step - loss: 0.5682 - accuracy: 0.8047
Epoch 14/20
18/18 [==============================] - 2s 113ms/step - loss: 0.5344 - accuracy: 0.8282
Epoch 15/20
18/18 [==============================] - 2s 113ms/step - loss: 0.4786 - accuracy: 0.8361
Epoch 16/20
18/18 [==============================] - 2s 120ms/step - loss: 0.4764 - accuracy: 0.8535
Epoch 17/20
18/18 [==============================] - 2s 112ms/step - loss: 0.4602 - accuracy: 0.8649
Epoch 18/20
18/18 [==============================] - 2s 113ms/step - loss: 0.4927 - accuracy: 0.8483
Epoch 19/20
18/18 [==============================] - 2s 113ms/step - loss: 0.5662 - accuracy: 0.8230
Epoch 20/20
18/18 [==============================] - 2s 113ms/step - loss: 0.5973 - accuracy: 0.8143
1/1 [==============================] - 0s 1ms/step - loss: 1.2371 - accuracy: 0.6789
Testing accuracy after 160 epochs: 67.9%


Epoch 1/20
18/18 [==============================] - 2s 116ms/step - loss: 0.6425 - accuracy: 0.8082
Epoch 2/20
18/18 [==============================] - 2s 114ms/step - loss: 0.8630 - accuracy: 0.7541
Epoch 3/20
18/18 [==============================] - 2s 113ms/step - loss: 0.8265 - accuracy: 0.7271
Epoch 4/20
18/18 [==============================] - 2s 114ms/step - loss: 0.7261 - accuracy: 0.7576
Epoch 5/20
18/18 [==============================] - 2s 114ms/step - loss: 0.6568 - accuracy: 0.7820
Epoch 6/20
18/18 [==============================] - 2s 115ms/step - loss: 0.6168 - accuracy: 0.7960
Epoch 7/20
18/18 [==============================] - 2s 117ms/step - loss: 0.6057 - accuracy: 0.7995
Epoch 8/20
18/18 [==============================] - 2s 115ms/step - loss: 0.5472 - accuracy: 0.8239
Epoch 9/20
18/18 [==============================] - 2s 113ms/step - loss: 0.5612 - accuracy: 0.8108
Epoch 10/20
18/18 [==============================] - 2s 114ms/step - loss: 0.5000 - accuracy: 0.8405
Epoch 11/20
18/18 [==============================] - 2s 116ms/step - loss: 0.5409 - accuracy: 0.8300
Epoch 12/20
18/18 [==============================] - 2s 114ms/step - loss: 0.5724 - accuracy: 0.8117
Epoch 13/20
18/18 [==============================] - 2s 113ms/step - loss: 0.5376 - accuracy: 0.8248
Epoch 14/20
18/18 [==============================] - 2s 114ms/step - loss: 0.5843 - accuracy: 0.7934
Epoch 15/20
18/18 [==============================] - 2s 119ms/step - loss: 0.5269 - accuracy: 0.8248
Epoch 16/20
18/18 [==============================] - 2s 136ms/step - loss: 0.4892 - accuracy: 0.8492
Epoch 17/20
18/18 [==============================] - 2s 119ms/step - loss: 0.4783 - accuracy: 0.8396
Epoch 18/20
18/18 [==============================] - 2s 116ms/step - loss: 0.5136 - accuracy: 0.8405
Epoch 19/20
18/18 [==============================] - 2s 117ms/step - loss: 0.4613 - accuracy: 0.8457
Epoch 20/20
18/18 [==============================] - 2s 123ms/step - loss: 0.4444 - accuracy: 0.8535
1/1 [==============================] - 0s 2ms/step - loss: 1.1809 - accuracy: 0.7258
Testing accuracy after 180 epochs: 72.6%


Epoch 1/20
18/18 [==============================] - 2s 115ms/step - loss: 0.4296 - accuracy: 0.8622
Epoch 2/20
18/18 [==============================] - 2s 125ms/step - loss: 0.5622 - accuracy: 0.8213
Epoch 3/20
18/18 [==============================] - 2s 118ms/step - loss: 0.4883 - accuracy: 0.8361
Epoch 4/20
18/18 [==============================] - 2s 116ms/step - loss: 0.4114 - accuracy: 0.8692
Epoch 5/20
18/18 [==============================] - 2s 115ms/step - loss: 0.4728 - accuracy: 0.8474
Epoch 6/20
18/18 [==============================] - 2s 114ms/step - loss: 0.5357 - accuracy: 0.8300
Epoch 7/20
18/18 [==============================] - 2s 115ms/step - loss: 0.4753 - accuracy: 0.8518
Epoch 8/20
18/18 [==============================] - 2s 116ms/step - loss: 0.5083 - accuracy: 0.8335
Epoch 9/20
18/18 [==============================] - 2s 117ms/step - loss: 0.4574 - accuracy: 0.8553
Epoch 10/20
18/18 [==============================] - 2s 114ms/step - loss: 0.4477 - accuracy: 0.8509
Epoch 11/20
18/18 [==============================] - 2s 114ms/step - loss: 0.3872 - accuracy: 0.8710
Epoch 12/20
18/18 [==============================] - 2s 115ms/step - loss: 0.3767 - accuracy: 0.8806
Epoch 13/20
18/18 [==============================] - 2s 115ms/step - loss: 0.4948 - accuracy: 0.8518
Epoch 14/20
18/18 [==============================] - 2s 116ms/step - loss: 0.5815 - accuracy: 0.8282
Epoch 15/20
18/18 [==============================] - 2s 115ms/step - loss: 0.4828 - accuracy: 0.8466
Epoch 16/20
18/18 [==============================] - 2s 115ms/step - loss: 0.4241 - accuracy: 0.8588
Epoch 17/20
18/18 [==============================] - 2s 115ms/step - loss: 0.4984 - accuracy: 0.8431
Epoch 18/20
18/18 [==============================] - 2s 117ms/step - loss: 0.3685 - accuracy: 0.8945
Epoch 19/20
18/18 [==============================] - 2s 116ms/step - loss: 0.4810 - accuracy: 0.8561
Epoch 20/20
18/18 [==============================] - 2s 113ms/step - loss: 0.7044 - accuracy: 0.7977
1/1 [==============================] - 0s 1ms/step - loss: 1.1721 - accuracy: 0.6923
Testing accuracy after 200 epochs: 69.2%


Epoch 1/20
18/18 [==============================] - 2s 115ms/step - loss: 0.5457 - accuracy: 0.8221
Epoch 2/20
18/18 [==============================] - 2s 114ms/step - loss: 0.4860 - accuracy: 0.8518
Epoch 3/20
18/18 [==============================] - 2s 116ms/step - loss: 0.4326 - accuracy: 0.8570
Epoch 4/20
18/18 [==============================] - 2s 121ms/step - loss: 0.3763 - accuracy: 0.8762
Epoch 5/20
18/18 [==============================] - 2s 115ms/step - loss: 0.4390 - accuracy: 0.8640
Epoch 6/20
18/18 [==============================] - 2s 117ms/step - loss: 0.4161 - accuracy: 0.8649
Epoch 7/20
18/18 [==============================] - 2s 116ms/step - loss: 0.3990 - accuracy: 0.8718
Epoch 8/20
18/18 [==============================] - 2s 116ms/step - loss: 0.3664 - accuracy: 0.8779
Epoch 9/20
18/18 [==============================] - 2s 115ms/step - loss: 0.3762 - accuracy: 0.8684
Epoch 10/20
18/18 [==============================] - 2s 116ms/step - loss: 0.4269 - accuracy: 0.8622
Epoch 11/20
18/18 [==============================] - 2s 116ms/step - loss: 0.3370 - accuracy: 0.8971
Epoch 12/20
18/18 [==============================] - 2s 115ms/step - loss: 0.3176 - accuracy: 0.9024
Epoch 13/20
18/18 [==============================] - 2s 114ms/step - loss: 0.3251 - accuracy: 0.9015
Epoch 14/20
18/18 [==============================] - 2s 114ms/step - loss: 0.3168 - accuracy: 0.8945
Epoch 15/20
18/18 [==============================] - 2s 117ms/step - loss: 0.2839 - accuracy: 0.9119
Epoch 16/20
18/18 [==============================] - 2s 119ms/step - loss: 0.2659 - accuracy: 0.9119
Epoch 17/20
18/18 [==============================] - 2s 116ms/step - loss: 0.2776 - accuracy: 0.9076
Epoch 18/20
18/18 [==============================] - 2s 116ms/step - loss: 0.3001 - accuracy: 0.9041
Epoch 19/20
18/18 [==============================] - 2s 116ms/step - loss: 0.3296 - accuracy: 0.9032
Epoch 20/20
18/18 [==============================] - 2s 114ms/step - loss: 0.3551 - accuracy: 0.8875
1/1 [==============================] - 0s 1ms/step - loss: 1.3045 - accuracy: 0.7358
Testing accuracy after 220 epochs: 73.6%


Epoch 1/20
18/18 [==============================] - 2s 121ms/step - loss: 0.3080 - accuracy: 0.8945
Epoch 2/20
18/18 [==============================] - 2s 112ms/step - loss: 0.3842 - accuracy: 0.8901
Epoch 3/20
18/18 [==============================] - 3s 149ms/step - loss: 0.4126 - accuracy: 0.8710
Epoch 4/20
18/18 [==============================] - 3s 148ms/step - loss: 0.3547 - accuracy: 0.8884
Epoch 5/20
18/18 [==============================] - 3s 150ms/step - loss: 0.2818 - accuracy: 0.9111
Epoch 6/20
18/18 [==============================] - 3s 174ms/step - loss: 0.2629 - accuracy: 0.9268
Epoch 7/20
18/18 [==============================] - 3s 152ms/step - loss: 0.2906 - accuracy: 0.9111
Epoch 8/20
18/18 [==============================] - 2s 116ms/step - loss: 0.3031 - accuracy: 0.9015
Epoch 9/20
18/18 [==============================] - 2s 120ms/step - loss: 0.3542 - accuracy: 0.8832
Epoch 10/20
18/18 [==============================] - 2s 114ms/step - loss: 0.3271 - accuracy: 0.8997
Epoch 11/20
18/18 [==============================] - 2s 113ms/step - loss: 0.5144 - accuracy: 0.8518
Epoch 12/20
18/18 [==============================] - 2s 114ms/step - loss: 0.5230 - accuracy: 0.8378
Epoch 13/20
18/18 [==============================] - 2s 113ms/step - loss: 0.4236 - accuracy: 0.8745
Epoch 14/20
18/18 [==============================] - 2s 115ms/step - loss: 0.3989 - accuracy: 0.8884
Epoch 15/20
18/18 [==============================] - 2s 120ms/step - loss: 0.3687 - accuracy: 0.8814
Epoch 16/20
18/18 [==============================] - 3s 149ms/step - loss: 0.3163 - accuracy: 0.8997
Epoch 17/20
18/18 [==============================] - 3s 158ms/step - loss: 0.3357 - accuracy: 0.8945
Epoch 18/20
18/18 [==============================] - 2s 119ms/step - loss: 0.3585 - accuracy: 0.8945
Epoch 19/20
18/18 [==============================] - 2s 115ms/step - loss: 0.5361 - accuracy: 0.8422
Epoch 20/20
18/18 [==============================] - 2s 122ms/step - loss: 0.5348 - accuracy: 0.8335
1/1 [==============================] - 0s 2ms/step - loss: 1.4235 - accuracy: 0.6823
Testing accuracy after 240 epochs: 68.2%


Epoch 1/20
18/18 [==============================] - 3s 145ms/step - loss: 0.6312 - accuracy: 0.8047
Epoch 2/20
18/18 [==============================] - 3s 150ms/step - loss: 0.5032 - accuracy: 0.8474
Epoch 3/20
18/18 [==============================] - 3s 169ms/step - loss: 0.4588 - accuracy: 0.8614
Epoch 4/20
18/18 [==============================] - 3s 162ms/step - loss: 0.4221 - accuracy: 0.8832
Epoch 5/20
18/18 [==============================] - 3s 149ms/step - loss: 0.3142 - accuracy: 0.9032
Epoch 6/20
18/18 [==============================] - 3s 173ms/step - loss: 0.3908 - accuracy: 0.8788
Epoch 7/20
18/18 [==============================] - 3s 145ms/step - loss: 0.4234 - accuracy: 0.8727
Epoch 8/20
18/18 [==============================] - 2s 136ms/step - loss: 0.3655 - accuracy: 0.8893
Epoch 9/20
18/18 [==============================] - 3s 157ms/step - loss: 0.3508 - accuracy: 0.9024
Epoch 10/20
18/18 [==============================] - 3s 146ms/step - loss: 0.3316 - accuracy: 0.8980
Epoch 11/20
18/18 [==============================] - 3s 166ms/step - loss: 0.3106 - accuracy: 0.9058
Epoch 12/20
18/18 [==============================] - 3s 156ms/step - loss: 0.3033 - accuracy: 0.9015
Epoch 13/20
18/18 [==============================] - 3s 152ms/step - loss: 0.2830 - accuracy: 0.9146
Epoch 14/20
18/18 [==============================] - 2s 117ms/step - loss: 0.2931 - accuracy: 0.9076
Epoch 15/20
18/18 [==============================] - 2s 116ms/step - loss: 0.2591 - accuracy: 0.9250
Epoch 16/20
18/18 [==============================] - 2s 115ms/step - loss: 0.3869 - accuracy: 0.9006
Epoch 17/20
18/18 [==============================] - 2s 116ms/step - loss: 0.4954 - accuracy: 0.8483
Epoch 18/20
18/18 [==============================] - 2s 115ms/step - loss: 0.3511 - accuracy: 0.8762
Epoch 19/20
18/18 [==============================] - 3s 139ms/step - loss: 0.2817 - accuracy: 0.9093
Epoch 20/20
18/18 [==============================] - 2s 123ms/step - loss: 0.2695 - accuracy: 0.9154
1/1 [==============================] - 0s 2ms/step - loss: 1.3146 - accuracy: 0.7458
Testing accuracy after 260 epochs: 74.6%


Epoch 1/20
18/18 [==============================] - 3s 145ms/step - loss: 0.2668 - accuracy: 0.9119
Epoch 2/20
18/18 [==============================] - 2s 118ms/step - loss: 0.2560 - accuracy: 0.9154
Epoch 3/20
18/18 [==============================] - 2s 129ms/step - loss: 0.2322 - accuracy: 0.9207
Epoch 4/20
18/18 [==============================] - 3s 150ms/step - loss: 0.2521 - accuracy: 0.9163
Epoch 5/20
18/18 [==============================] - 3s 154ms/step - loss: 0.3949 - accuracy: 0.8788
Epoch 6/20
18/18 [==============================] - 3s 183ms/step - loss: 0.3254 - accuracy: 0.9032
Epoch 7/20
18/18 [==============================] - 2s 120ms/step - loss: 0.2781 - accuracy: 0.9111
Epoch 8/20
18/18 [==============================] - 2s 115ms/step - loss: 0.2564 - accuracy: 0.9154
Epoch 9/20
18/18 [==============================] - 2s 115ms/step - loss: 0.2680 - accuracy: 0.9172
Epoch 10/20
18/18 [==============================] - 3s 147ms/step - loss: 0.2493 - accuracy: 0.9285
Epoch 11/20
18/18 [==============================] - 3s 149ms/step - loss: 0.2358 - accuracy: 0.9303
Epoch 12/20
18/18 [==============================] - 2s 136ms/step - loss: 0.2073 - accuracy: 0.9372
Epoch 13/20
18/18 [==============================] - 3s 143ms/step - loss: 0.2034 - accuracy: 0.9364
Epoch 14/20
18/18 [==============================] - 2s 139ms/step - loss: 0.2181 - accuracy: 0.9355
Epoch 15/20
18/18 [==============================] - 2s 122ms/step - loss: 0.2117 - accuracy: 0.9337
Epoch 16/20
18/18 [==============================] - 3s 154ms/step - loss: 0.2339 - accuracy: 0.9294
Epoch 17/20
18/18 [==============================] - 3s 152ms/step - loss: 0.2059 - accuracy: 0.9355
Epoch 18/20
18/18 [==============================] - 3s 150ms/step - loss: 0.2267 - accuracy: 0.9241
Epoch 19/20
18/18 [==============================] - 3s 156ms/step - loss: 0.2101 - accuracy: 0.9364
Epoch 20/20
18/18 [==============================] - 3s 149ms/step - loss: 0.2392 - accuracy: 0.9241
1/1 [==============================] - 0s 1ms/step - loss: 1.4929 - accuracy: 0.7559
Testing accuracy after 280 epochs: 75.6%


Epoch 1/20
18/18 [==============================] - 2s 134ms/step - loss: 0.1904 - accuracy: 0.9451
Epoch 2/20
18/18 [==============================] - 2s 122ms/step - loss: 0.2163 - accuracy: 0.9303
Epoch 3/20
18/18 [==============================] - 2s 120ms/step - loss: 0.2736 - accuracy: 0.9311
Epoch 4/20
18/18 [==============================] - 2s 130ms/step - loss: 0.3340 - accuracy: 0.8954
Epoch 5/20
18/18 [==============================] - 2s 115ms/step - loss: 0.2771 - accuracy: 0.9137
Epoch 6/20
18/18 [==============================] - 2s 130ms/step - loss: 0.2455 - accuracy: 0.9180
Epoch 7/20
18/18 [==============================] - 2s 119ms/step - loss: 0.2334 - accuracy: 0.9285
Epoch 8/20
18/18 [==============================] - 2s 116ms/step - loss: 0.2422 - accuracy: 0.9180
Epoch 9/20
18/18 [==============================] - 2s 116ms/step - loss: 0.3279 - accuracy: 0.8980
Epoch 10/20
18/18 [==============================] - 2s 128ms/step - loss: 0.3010 - accuracy: 0.9207
Epoch 11/20
18/18 [==============================] - 2s 118ms/step - loss: 0.3095 - accuracy: 0.9102
Epoch 12/20
18/18 [==============================] - 2s 122ms/step - loss: 0.3220 - accuracy: 0.9058
Epoch 13/20
18/18 [==============================] - 2s 130ms/step - loss: 0.2498 - accuracy: 0.9215
Epoch 14/20
18/18 [==============================] - 2s 116ms/step - loss: 0.2988 - accuracy: 0.9128
Epoch 15/20
18/18 [==============================] - 2s 135ms/step - loss: 0.3638 - accuracy: 0.8971
Epoch 16/20
18/18 [==============================] - 2s 120ms/step - loss: 0.2579 - accuracy: 0.9119
Epoch 17/20
18/18 [==============================] - 2s 114ms/step - loss: 0.2414 - accuracy: 0.9276
Epoch 18/20
18/18 [==============================] - 2s 120ms/step - loss: 0.1923 - accuracy: 0.9355
Epoch 19/20
18/18 [==============================] - 2s 120ms/step - loss: 0.1731 - accuracy: 0.9459
Epoch 20/20
18/18 [==============================] - 3s 145ms/step - loss: 0.3045 - accuracy: 0.9163
1/1 [==============================] - 0s 1ms/step - loss: 1.5787 - accuracy: 0.7224
Testing accuracy after 300 epochs: 72.2%



The model trained for 280 epochs performed best on held out test data (0.8% accuracy)

# We see that training accuracy appears very high at ths point (90-95%) while testing accuracy is around 75%
# We also notice that test accuracy is roughly increasing with more training
# Therefore we launch a second round of training and testing, just starting with the latest model

for testing_index in range(number_of_models_to_test):
    testing_index = testing_index # + number_of_models_to_test # to account for previously trained model indices
    model.fit(train_X, one_hot_train_Y, epochs=epochs_between_tests, batch_size=input_batch_size)
    _, accuracy = model.evaluate(test_X, one_hot_test_Y, batch_size=one_hot_test_Y.shape[0])
    number_of_epochs_trained_for_model = (testing_index+1) * epochs_between_tests
    epochs_trained.append(number_of_epochs_trained_for_model)
    print("Testing accuracy after {} additional epochs: {:.1f}%\n\n".format(number_of_epochs_trained_for_model, 100*accuracy))
    all_test_model_accuracies[testing_index] = accuracy
    model_filepath = "{}/lstm_walker_recognition_model_{}.h5".format(folder, testing_index)
    model.save(model_filepath)
    model_filepaths.append(model_filepath)

max_model_index = np.argmax(all_test_model_accuracies)
best_model_filepath = model_filepaths[max_model_index]
best_model_test_accuracy = all_test_model_accuracies[max_model_index]
best_model_epochs_trained = epochs_trained[max_model_index]
print("\nThe model trained for {} additional epochs performed best on held out test data ({:.1f}% accuracy)".format(best_model_epochs_trained, 100*best_model_test_accuracy))

Epoch 1/20
18/18 [==============================] - 2s 122ms/step - loss: 0.2653 - accuracy: 0.9137
Epoch 2/20
18/18 [==============================] - 3s 146ms/step - loss: 0.2931 - accuracy: 0.9041
Epoch 3/20
18/18 [==============================] - 2s 119ms/step - loss: 0.2427 - accuracy: 0.9198
Epoch 4/20
18/18 [==============================] - 2s 124ms/step - loss: 0.3595 - accuracy: 0.8884
Epoch 5/20
18/18 [==============================] - 3s 141ms/step - loss: 0.2455 - accuracy: 0.9180
Epoch 6/20
18/18 [==============================] - 2s 117ms/step - loss: 0.2522 - accuracy: 0.9303
Epoch 7/20
18/18 [==============================] - 2s 116ms/step - loss: 0.2063 - accuracy: 0.9372
Epoch 8/20
18/18 [==============================] - 2s 120ms/step - loss: 0.2382 - accuracy: 0.9329
Epoch 9/20
18/18 [==============================] - 2s 119ms/step - loss: 0.2413 - accuracy: 0.9320
Epoch 10/20
18/18 [==============================] - 2s 119ms/step - loss: 0.2273 - accuracy: 0.9294
Epoch 11/20
18/18 [==============================] - 2s 115ms/step - loss: 0.2372 - accuracy: 0.9276
Epoch 12/20
18/18 [==============================] - 2s 116ms/step - loss: 0.2200 - accuracy: 0.9337
Epoch 13/20
18/18 [==============================] - 2s 115ms/step - loss: 0.2382 - accuracy: 0.9215
Epoch 14/20
18/18 [==============================] - 2s 120ms/step - loss: 0.1781 - accuracy: 0.9494
Epoch 15/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1636 - accuracy: 0.9468
Epoch 16/20
18/18 [==============================] - 2s 122ms/step - loss: 0.1820 - accuracy: 0.9416
Epoch 17/20
18/18 [==============================] - 2s 121ms/step - loss: 0.1550 - accuracy: 0.9520
Epoch 18/20
18/18 [==============================] - 2s 121ms/step - loss: 0.1614 - accuracy: 0.9451
Epoch 19/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1665 - accuracy: 0.9459
Epoch 20/20
18/18 [==============================] - 2s 114ms/step - loss: 0.1754 - accuracy: 0.9494
1/1 [==============================] - 0s 2ms/step - loss: 1.6940 - accuracy: 0.7492
Testing accuracy after 20 additional epochs: 74.9%


Epoch 1/20
18/18 [==============================] - 2s 117ms/step - loss: 0.4720 - accuracy: 0.8910
Epoch 2/20
18/18 [==============================] - 2s 115ms/step - loss: 0.3330 - accuracy: 0.9119
Epoch 3/20
18/18 [==============================] - 2s 115ms/step - loss: 0.3294 - accuracy: 0.9006
Epoch 4/20
18/18 [==============================] - 2s 118ms/step - loss: 0.3366 - accuracy: 0.9085
Epoch 5/20
18/18 [==============================] - 2s 115ms/step - loss: 0.3554 - accuracy: 0.9015
Epoch 6/20
18/18 [==============================] - 2s 115ms/step - loss: 0.3336 - accuracy: 0.8989
Epoch 7/20
18/18 [==============================] - 2s 132ms/step - loss: 0.2703 - accuracy: 0.9146
Epoch 8/20
18/18 [==============================] - 2s 118ms/step - loss: 0.2237 - accuracy: 0.9337
Epoch 9/20
18/18 [==============================] - 2s 117ms/step - loss: 0.2057 - accuracy: 0.9398
Epoch 10/20
18/18 [==============================] - 2s 117ms/step - loss: 0.1550 - accuracy: 0.9555
Epoch 11/20
18/18 [==============================] - 2s 117ms/step - loss: 0.1929 - accuracy: 0.9372
Epoch 12/20
18/18 [==============================] - 2s 117ms/step - loss: 0.2344 - accuracy: 0.9250
Epoch 13/20
18/18 [==============================] - 2s 116ms/step - loss: 0.2028 - accuracy: 0.9364
Epoch 14/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1931 - accuracy: 0.9416
Epoch 15/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1423 - accuracy: 0.9520
Epoch 16/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1387 - accuracy: 0.9520
Epoch 17/20
18/18 [==============================] - 2s 117ms/step - loss: 0.1370 - accuracy: 0.9573
Epoch 18/20
18/18 [==============================] - 2s 116ms/step - loss: 0.1599 - accuracy: 0.9494
Epoch 19/20
18/18 [==============================] - 2s 115ms/step - loss: 0.3934 - accuracy: 0.9154
Epoch 20/20
18/18 [==============================] - 2s 114ms/step - loss: 0.6137 - accuracy: 0.8405
1/1 [==============================] - 0s 2ms/step - loss: 1.2670 - accuracy: 0.7057
Testing accuracy after 40 additional epochs: 70.6%


Epoch 1/20
18/18 [==============================] - 2s 123ms/step - loss: 0.4121 - accuracy: 0.8701
Epoch 2/20
18/18 [==============================] - 2s 117ms/step - loss: 0.2409 - accuracy: 0.9137
Epoch 3/20
18/18 [==============================] - 2s 123ms/step - loss: 0.2081 - accuracy: 0.9285
Epoch 4/20
18/18 [==============================] - 2s 116ms/step - loss: 0.2078 - accuracy: 0.9372
Epoch 5/20
18/18 [==============================] - 2s 120ms/step - loss: 0.1731 - accuracy: 0.9468
Epoch 6/20
18/18 [==============================] - 2s 116ms/step - loss: 0.1444 - accuracy: 0.9555
Epoch 7/20
18/18 [==============================] - 2s 116ms/step - loss: 0.1458 - accuracy: 0.9512
Epoch 8/20
18/18 [==============================] - 2s 116ms/step - loss: 0.1444 - accuracy: 0.9599
Epoch 9/20
18/18 [==============================] - 2s 120ms/step - loss: 0.1312 - accuracy: 0.9564
Epoch 10/20
18/18 [==============================] - 2s 118ms/step - loss: 0.1307 - accuracy: 0.9582
Epoch 11/20
18/18 [==============================] - 2s 118ms/step - loss: 0.1359 - accuracy: 0.9564
Epoch 12/20
18/18 [==============================] - 2s 117ms/step - loss: 0.1308 - accuracy: 0.9555
Epoch 13/20
18/18 [==============================] - 2s 117ms/step - loss: 0.1270 - accuracy: 0.9634
Epoch 14/20
18/18 [==============================] - 3s 144ms/step - loss: 0.3530 - accuracy: 0.9119
Epoch 15/20
18/18 [==============================] - 2s 124ms/step - loss: 0.5290 - accuracy: 0.8544
Epoch 16/20
18/18 [==============================] - 2s 119ms/step - loss: 0.3789 - accuracy: 0.8779
Epoch 17/20
18/18 [==============================] - 2s 120ms/step - loss: 0.2999 - accuracy: 0.9024
Epoch 18/20
18/18 [==============================] - 2s 117ms/step - loss: 0.2957 - accuracy: 0.9041
Epoch 19/20
18/18 [==============================] - 2s 117ms/step - loss: 0.2324 - accuracy: 0.9259
Epoch 20/20
18/18 [==============================] - 2s 117ms/step - loss: 0.1775 - accuracy: 0.9425
1/1 [==============================] - 0s 1ms/step - loss: 1.4669 - accuracy: 0.7625
Testing accuracy after 60 additional epochs: 76.3%


Epoch 1/20
18/18 [==============================] - 2s 123ms/step - loss: 0.1809 - accuracy: 0.9425
Epoch 2/20
18/18 [==============================] - 2s 118ms/step - loss: 0.2290 - accuracy: 0.9355
Epoch 3/20
18/18 [==============================] - 2s 121ms/step - loss: 0.4448 - accuracy: 0.8867
Epoch 4/20
18/18 [==============================] - 2s 119ms/step - loss: 0.3926 - accuracy: 0.8753
Epoch 5/20
18/18 [==============================] - 2s 119ms/step - loss: 0.2940 - accuracy: 0.9198
Epoch 6/20
18/18 [==============================] - 2s 117ms/step - loss: 0.3013 - accuracy: 0.9224
Epoch 7/20
18/18 [==============================] - 2s 120ms/step - loss: 0.4075 - accuracy: 0.8901
Epoch 8/20
18/18 [==============================] - 2s 119ms/step - loss: 0.3126 - accuracy: 0.9085
Epoch 9/20
18/18 [==============================] - 2s 117ms/step - loss: 0.2510 - accuracy: 0.9294
Epoch 10/20
18/18 [==============================] - 2s 120ms/step - loss: 0.2231 - accuracy: 0.9355
Epoch 11/20
18/18 [==============================] - 2s 117ms/step - loss: 0.1821 - accuracy: 0.9425
Epoch 12/20
18/18 [==============================] - 2s 120ms/step - loss: 0.1946 - accuracy: 0.9390
Epoch 13/20
18/18 [==============================] - 2s 117ms/step - loss: 0.1829 - accuracy: 0.9503
Epoch 14/20
18/18 [==============================] - 2s 118ms/step - loss: 0.1652 - accuracy: 0.9538
Epoch 15/20
18/18 [==============================] - 2s 126ms/step - loss: 0.1701 - accuracy: 0.9512
Epoch 16/20
18/18 [==============================] - 2s 120ms/step - loss: 0.1868 - accuracy: 0.9433
Epoch 17/20
18/18 [==============================] - 2s 117ms/step - loss: 0.1347 - accuracy: 0.9643
Epoch 18/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1549 - accuracy: 0.9555
Epoch 19/20
18/18 [==============================] - 2s 113ms/step - loss: 0.1436 - accuracy: 0.9608
Epoch 20/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1250 - accuracy: 0.9590
1/1 [==============================] - 0s 3ms/step - loss: 1.5178 - accuracy: 0.7692
Testing accuracy after 80 additional epochs: 76.9%


Epoch 1/20
18/18 [==============================] - 2s 114ms/step - loss: 0.1335 - accuracy: 0.9582
Epoch 2/20
18/18 [==============================] - 2s 114ms/step - loss: 0.1241 - accuracy: 0.9643
Epoch 3/20
18/18 [==============================] - 2s 113ms/step - loss: 0.1531 - accuracy: 0.9555
Epoch 4/20
18/18 [==============================] - 2s 113ms/step - loss: 0.1524 - accuracy: 0.9547
Epoch 5/20
18/18 [==============================] - 2s 113ms/step - loss: 0.1366 - accuracy: 0.9538
Epoch 6/20
18/18 [==============================] - 2s 113ms/step - loss: 0.1495 - accuracy: 0.9520
Epoch 7/20
18/18 [==============================] - 2s 112ms/step - loss: 0.1353 - accuracy: 0.9564
Epoch 8/20
18/18 [==============================] - 2s 113ms/step - loss: 0.3099 - accuracy: 0.9241
Epoch 9/20
18/18 [==============================] - 2s 116ms/step - loss: 0.5383 - accuracy: 0.8701
Epoch 10/20
18/18 [==============================] - 2s 113ms/step - loss: 0.4158 - accuracy: 0.8858
Epoch 11/20
18/18 [==============================] - 2s 113ms/step - loss: 0.3272 - accuracy: 0.9050
Epoch 12/20
18/18 [==============================] - 2s 114ms/step - loss: 0.2184 - accuracy: 0.9294
Epoch 13/20
18/18 [==============================] - 2s 120ms/step - loss: 0.1883 - accuracy: 0.9416
Epoch 14/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1587 - accuracy: 0.9459
Epoch 15/20
18/18 [==============================] - 2s 114ms/step - loss: 0.1511 - accuracy: 0.9547
Epoch 16/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1228 - accuracy: 0.9616
Epoch 17/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1417 - accuracy: 0.9573
Epoch 18/20
18/18 [==============================] - 2s 113ms/step - loss: 0.1549 - accuracy: 0.9590
Epoch 19/20
18/18 [==============================] - 2s 113ms/step - loss: 0.1562 - accuracy: 0.9564
Epoch 20/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1860 - accuracy: 0.9486
1/1 [==============================] - 0s 2ms/step - loss: 1.7033 - accuracy: 0.7592
Testing accuracy after 100 additional epochs: 75.9%


Epoch 1/20
18/18 [==============================] - 2s 114ms/step - loss: 0.2180 - accuracy: 0.9355
Epoch 2/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1489 - accuracy: 0.9608
Epoch 3/20
18/18 [==============================] - 2s 116ms/step - loss: 0.1711 - accuracy: 0.9494
Epoch 4/20
18/18 [==============================] - 2s 115ms/step - loss: 0.2529 - accuracy: 0.9329
Epoch 5/20
18/18 [==============================] - 2s 114ms/step - loss: 0.3105 - accuracy: 0.9154
Epoch 6/20
18/18 [==============================] - 2s 114ms/step - loss: 0.2234 - accuracy: 0.9303
Epoch 7/20
18/18 [==============================] - 3s 141ms/step - loss: 0.1948 - accuracy: 0.9372
Epoch 8/20
18/18 [==============================] - 2s 129ms/step - loss: 0.1723 - accuracy: 0.9512
Epoch 9/20
18/18 [==============================] - 2s 117ms/step - loss: 0.1558 - accuracy: 0.9564
Epoch 10/20
18/18 [==============================] - 2s 116ms/step - loss: 0.1316 - accuracy: 0.9564
Epoch 11/20
18/18 [==============================] - 2s 116ms/step - loss: 0.1442 - accuracy: 0.9520
Epoch 12/20
18/18 [==============================] - 2s 116ms/step - loss: 0.1333 - accuracy: 0.9573
Epoch 13/20
18/18 [==============================] - 2s 116ms/step - loss: 0.1055 - accuracy: 0.9686
Epoch 14/20
18/18 [==============================] - 2s 120ms/step - loss: 0.1175 - accuracy: 0.9677
Epoch 15/20
18/18 [==============================] - 2s 124ms/step - loss: 0.2069 - accuracy: 0.9425
Epoch 16/20
18/18 [==============================] - 2s 116ms/step - loss: 0.1753 - accuracy: 0.9459
Epoch 17/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1742 - accuracy: 0.9468
Epoch 18/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1590 - accuracy: 0.9503
Epoch 19/20
18/18 [==============================] - 2s 116ms/step - loss: 0.1694 - accuracy: 0.9486
Epoch 20/20
18/18 [==============================] - 2s 117ms/step - loss: 0.1498 - accuracy: 0.9442
1/1 [==============================] - 0s 2ms/step - loss: 1.6868 - accuracy: 0.7492
Testing accuracy after 120 additional epochs: 74.9%


Epoch 1/20
18/18 [==============================] - 2s 118ms/step - loss: 0.1535 - accuracy: 0.9494
Epoch 2/20
18/18 [==============================] - 2s 115ms/step - loss: 0.4269 - accuracy: 0.9032
Epoch 3/20
18/18 [==============================] - 2s 115ms/step - loss: 0.3909 - accuracy: 0.8980
Epoch 4/20
18/18 [==============================] - 2s 116ms/step - loss: 0.3581 - accuracy: 0.8980
Epoch 5/20
18/18 [==============================] - 2s 116ms/step - loss: 0.2286 - accuracy: 0.9224
Epoch 6/20
18/18 [==============================] - 2s 114ms/step - loss: 0.2088 - accuracy: 0.9320
Epoch 7/20
18/18 [==============================] - 2s 115ms/step - loss: 0.6760 - accuracy: 0.8492
Epoch 8/20
18/18 [==============================] - 3s 152ms/step - loss: 1.3396 - accuracy: 0.6818
Epoch 9/20
18/18 [==============================] - 2s 117ms/step - loss: 0.7833 - accuracy: 0.7428
Epoch 10/20
18/18 [==============================] - 2s 115ms/step - loss: 0.5284 - accuracy: 0.8152
Epoch 11/20
18/18 [==============================] - 2s 115ms/step - loss: 0.4476 - accuracy: 0.8431
Epoch 12/20
18/18 [==============================] - 2s 124ms/step - loss: 0.3612 - accuracy: 0.8875
Epoch 13/20
18/18 [==============================] - 2s 115ms/step - loss: 0.3148 - accuracy: 0.8963
Epoch 14/20
18/18 [==============================] - 2s 115ms/step - loss: 0.2772 - accuracy: 0.9050
Epoch 15/20
18/18 [==============================] - 2s 115ms/step - loss: 0.2429 - accuracy: 0.9215
Epoch 16/20
18/18 [==============================] - 2s 113ms/step - loss: 0.2700 - accuracy: 0.9085
Epoch 17/20
18/18 [==============================] - 2s 115ms/step - loss: 0.2609 - accuracy: 0.9250
Epoch 18/20
18/18 [==============================] - 2s 113ms/step - loss: 0.2311 - accuracy: 0.9268
Epoch 19/20
18/18 [==============================] - 2s 112ms/step - loss: 0.2360 - accuracy: 0.9250
Epoch 20/20
18/18 [==============================] - 2s 114ms/step - loss: 0.2463 - accuracy: 0.9303
1/1 [==============================] - 0s 1ms/step - loss: 1.3512 - accuracy: 0.7659
Testing accuracy after 140 additional epochs: 76.6%


Epoch 1/20
18/18 [==============================] - 2s 117ms/step - loss: 0.2132 - accuracy: 0.9241
Epoch 2/20
18/18 [==============================] - 2s 114ms/step - loss: 0.2053 - accuracy: 0.9381
Epoch 3/20
18/18 [==============================] - 2s 116ms/step - loss: 0.1722 - accuracy: 0.9425
Epoch 4/20
18/18 [==============================] - 2s 114ms/step - loss: 0.1778 - accuracy: 0.9425
Epoch 5/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1511 - accuracy: 0.9555
Epoch 6/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1785 - accuracy: 0.9364
Epoch 7/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1344 - accuracy: 0.9616
Epoch 8/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1417 - accuracy: 0.9529
Epoch 9/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1404 - accuracy: 0.9555
Epoch 10/20
18/18 [==============================] - 2s 116ms/step - loss: 0.1403 - accuracy: 0.9520
Epoch 11/20
18/18 [==============================] - 2s 113ms/step - loss: 0.1204 - accuracy: 0.9608
Epoch 12/20
18/18 [==============================] - 2s 120ms/step - loss: 0.1076 - accuracy: 0.9651
Epoch 13/20
18/18 [==============================] - 2s 114ms/step - loss: 0.1020 - accuracy: 0.9660
Epoch 14/20
18/18 [==============================] - 2s 116ms/step - loss: 0.0979 - accuracy: 0.9730
Epoch 15/20
18/18 [==============================] - 2s 117ms/step - loss: 0.1199 - accuracy: 0.9590
Epoch 16/20
18/18 [==============================] - 2s 113ms/step - loss: 0.1118 - accuracy: 0.9634
Epoch 17/20
18/18 [==============================] - 2s 115ms/step - loss: 0.2261 - accuracy: 0.9477
Epoch 18/20
18/18 [==============================] - 2s 113ms/step - loss: 0.1801 - accuracy: 0.9494
Epoch 19/20
18/18 [==============================] - 2s 113ms/step - loss: 0.1532 - accuracy: 0.9520
Epoch 20/20
18/18 [==============================] - 3s 143ms/step - loss: 0.2394 - accuracy: 0.9268
1/1 [==============================] - 0s 1ms/step - loss: 1.4673 - accuracy: 0.7659
Testing accuracy after 160 additional epochs: 76.6%


Epoch 1/20
18/18 [==============================] - 2s 120ms/step - loss: 0.1924 - accuracy: 0.9486
Epoch 2/20
18/18 [==============================] - 2s 114ms/step - loss: 0.1182 - accuracy: 0.9634
Epoch 3/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1960 - accuracy: 0.9477
Epoch 4/20
18/18 [==============================] - 2s 114ms/step - loss: 0.5392 - accuracy: 0.8867
Epoch 5/20
18/18 [==============================] - 2s 113ms/step - loss: 0.4923 - accuracy: 0.8596
Epoch 6/20
18/18 [==============================] - 2s 115ms/step - loss: 0.3099 - accuracy: 0.9111
Epoch 7/20
18/18 [==============================] - 2s 119ms/step - loss: 0.2821 - accuracy: 0.9085
Epoch 8/20
18/18 [==============================] - 2s 113ms/step - loss: 0.2945 - accuracy: 0.9050
Epoch 9/20
18/18 [==============================] - 2s 126ms/step - loss: 0.2454 - accuracy: 0.9241
Epoch 10/20
18/18 [==============================] - 2s 114ms/step - loss: 0.2056 - accuracy: 0.9442
Epoch 11/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1816 - accuracy: 0.9459
Epoch 12/20
18/18 [==============================] - 2s 115ms/step - loss: 0.2073 - accuracy: 0.9337
Epoch 13/20
18/18 [==============================] - 2s 117ms/step - loss: 0.1898 - accuracy: 0.9381
Epoch 14/20
18/18 [==============================] - 2s 125ms/step - loss: 0.2211 - accuracy: 0.9459
Epoch 15/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1724 - accuracy: 0.9433
Epoch 16/20
18/18 [==============================] - 2s 116ms/step - loss: 0.2146 - accuracy: 0.9459
Epoch 17/20
18/18 [==============================] - 2s 116ms/step - loss: 0.2113 - accuracy: 0.9329
Epoch 18/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1631 - accuracy: 0.9538
Epoch 19/20
18/18 [==============================] - 2s 114ms/step - loss: 0.1445 - accuracy: 0.9564
Epoch 20/20
18/18 [==============================] - 2s 114ms/step - loss: 0.1252 - accuracy: 0.9616
1/1 [==============================] - 0s 1ms/step - loss: 1.5313 - accuracy: 0.7726
Testing accuracy after 180 additional epochs: 77.3%


Epoch 1/20
18/18 [==============================] - 2s 116ms/step - loss: 0.1175 - accuracy: 0.9625
Epoch 2/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1323 - accuracy: 0.9634
Epoch 3/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1048 - accuracy: 0.9677
Epoch 4/20
18/18 [==============================] - 2s 114ms/step - loss: 0.0968 - accuracy: 0.9712
Epoch 5/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1207 - accuracy: 0.9599
Epoch 6/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1292 - accuracy: 0.9608
Epoch 7/20
18/18 [==============================] - 2s 123ms/step - loss: 0.1522 - accuracy: 0.9643
Epoch 8/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1983 - accuracy: 0.9477
Epoch 9/20
18/18 [==============================] - 2s 114ms/step - loss: 0.1507 - accuracy: 0.9573
Epoch 10/20
18/18 [==============================] - 2s 117ms/step - loss: 0.1525 - accuracy: 0.9538
Epoch 11/20
18/18 [==============================] - 2s 117ms/step - loss: 0.1190 - accuracy: 0.9599
Epoch 12/20
18/18 [==============================] - 2s 119ms/step - loss: 0.1181 - accuracy: 0.9590
Epoch 13/20
18/18 [==============================] - 2s 115ms/step - loss: 0.0940 - accuracy: 0.9651
Epoch 14/20
18/18 [==============================] - 2s 115ms/step - loss: 0.0802 - accuracy: 0.9782
Epoch 15/20
18/18 [==============================] - 2s 114ms/step - loss: 0.1019 - accuracy: 0.9651
Epoch 16/20
18/18 [==============================] - 2s 116ms/step - loss: 0.1446 - accuracy: 0.9625
Epoch 17/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1408 - accuracy: 0.9625
Epoch 18/20
18/18 [==============================] - 2s 113ms/step - loss: 0.1404 - accuracy: 0.9555
Epoch 19/20
18/18 [==============================] - 2s 114ms/step - loss: 0.1262 - accuracy: 0.9599
Epoch 20/20
18/18 [==============================] - 2s 117ms/step - loss: 0.2231 - accuracy: 0.9486
1/1 [==============================] - 0s 2ms/step - loss: 1.5303 - accuracy: 0.7860
Testing accuracy after 200 additional epochs: 78.6%


Epoch 1/20
18/18 [==============================] - 2s 116ms/step - loss: 0.1953 - accuracy: 0.9512
Epoch 2/20
18/18 [==============================] - 2s 117ms/step - loss: 0.1477 - accuracy: 0.9590
Epoch 3/20
18/18 [==============================] - 2s 124ms/step - loss: 0.1226 - accuracy: 0.9660
Epoch 4/20
18/18 [==============================] - 2s 117ms/step - loss: 0.1641 - accuracy: 0.9547
Epoch 5/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1381 - accuracy: 0.9582
Epoch 6/20
18/18 [==============================] - 3s 142ms/step - loss: 0.1076 - accuracy: 0.9651
Epoch 7/20
18/18 [==============================] - 3s 149ms/step - loss: 0.1074 - accuracy: 0.9669
Epoch 8/20
18/18 [==============================] - 3s 155ms/step - loss: 0.0840 - accuracy: 0.9730
Epoch 9/20
18/18 [==============================] - 3s 149ms/step - loss: 0.0988 - accuracy: 0.9712
Epoch 10/20
18/18 [==============================] - 3s 149ms/step - loss: 0.1146 - accuracy: 0.9677
Epoch 11/20
18/18 [==============================] - 3s 148ms/step - loss: 0.1141 - accuracy: 0.9660
Epoch 12/20
18/18 [==============================] - 3s 178ms/step - loss: 0.0890 - accuracy: 0.9730
Epoch 13/20
18/18 [==============================] - 3s 157ms/step - loss: 0.0916 - accuracy: 0.9686
Epoch 14/20
18/18 [==============================] - 2s 137ms/step - loss: 0.0988 - accuracy: 0.9677
Epoch 15/20
18/18 [==============================] - 2s 130ms/step - loss: 0.1090 - accuracy: 0.9730
Epoch 16/20
18/18 [==============================] - 2s 136ms/step - loss: 0.0870 - accuracy: 0.9686
Epoch 17/20
18/18 [==============================] - 2s 136ms/step - loss: 0.1155 - accuracy: 0.9660
Epoch 18/20
18/18 [==============================] - 2s 132ms/step - loss: 0.1740 - accuracy: 0.9651
Epoch 19/20
18/18 [==============================] - 2s 132ms/step - loss: 0.2875 - accuracy: 0.9224
Epoch 20/20
18/18 [==============================] - 2s 130ms/step - loss: 0.2921 - accuracy: 0.9154
1/1 [==============================] - 0s 3ms/step - loss: 1.4087 - accuracy: 0.7492
Testing accuracy after 220 additional epochs: 74.9%


Epoch 1/20
18/18 [==============================] - 2s 133ms/step - loss: 0.2358 - accuracy: 0.9346
Epoch 2/20
18/18 [==============================] - 2s 132ms/step - loss: 0.1797 - accuracy: 0.9407
Epoch 3/20
18/18 [==============================] - 2s 126ms/step - loss: 0.1799 - accuracy: 0.9398
Epoch 4/20
18/18 [==============================] - 2s 129ms/step - loss: 0.1457 - accuracy: 0.9590
Epoch 5/20
18/18 [==============================] - 2s 118ms/step - loss: 0.1936 - accuracy: 0.9468
Epoch 6/20
18/18 [==============================] - 2s 118ms/step - loss: 0.1397 - accuracy: 0.9538
Epoch 7/20
18/18 [==============================] - 2s 119ms/step - loss: 0.1254 - accuracy: 0.9529
Epoch 8/20
18/18 [==============================] - 2s 118ms/step - loss: 0.1023 - accuracy: 0.9704
Epoch 9/20
18/18 [==============================] - 2s 123ms/step - loss: 0.0932 - accuracy: 0.9721
Epoch 10/20
18/18 [==============================] - 2s 123ms/step - loss: 0.1044 - accuracy: 0.9660
Epoch 11/20
18/18 [==============================] - 2s 119ms/step - loss: 0.0895 - accuracy: 0.9695
Epoch 12/20
18/18 [==============================] - 2s 124ms/step - loss: 0.1011 - accuracy: 0.9643
Epoch 13/20
18/18 [==============================] - 2s 115ms/step - loss: 0.0777 - accuracy: 0.9773
Epoch 14/20
18/18 [==============================] - 2s 114ms/step - loss: 0.0924 - accuracy: 0.9695
Epoch 15/20
18/18 [==============================] - 2s 114ms/step - loss: 0.0729 - accuracy: 0.9756
Epoch 16/20
18/18 [==============================] - 2s 118ms/step - loss: 0.0978 - accuracy: 0.9660
Epoch 17/20
18/18 [==============================] - 2s 116ms/step - loss: 0.0983 - accuracy: 0.9686
Epoch 18/20
18/18 [==============================] - 2s 116ms/step - loss: 0.0812 - accuracy: 0.9782
Epoch 19/20
18/18 [==============================] - 2s 116ms/step - loss: 0.0845 - accuracy: 0.9730
Epoch 20/20
18/18 [==============================] - 2s 117ms/step - loss: 0.0940 - accuracy: 0.9721
1/1 [==============================] - 0s 2ms/step - loss: 1.6166 - accuracy: 0.7659
Testing accuracy after 240 additional epochs: 76.6%


Epoch 1/20
18/18 [==============================] - 2s 117ms/step - loss: 0.0983 - accuracy: 0.9704
Epoch 2/20
18/18 [==============================] - 2s 116ms/step - loss: 0.1088 - accuracy: 0.9651
Epoch 3/20
18/18 [==============================] - 2s 114ms/step - loss: 0.1896 - accuracy: 0.9599
Epoch 4/20
18/18 [==============================] - 2s 114ms/step - loss: 0.1741 - accuracy: 0.9529
Epoch 5/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1316 - accuracy: 0.9599
Epoch 6/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1243 - accuracy: 0.9599
Epoch 7/20
18/18 [==============================] - 2s 114ms/step - loss: 0.1744 - accuracy: 0.9477
Epoch 8/20
18/18 [==============================] - 2s 117ms/step - loss: 0.1853 - accuracy: 0.9477
Epoch 9/20
18/18 [==============================] - 2s 116ms/step - loss: 0.1744 - accuracy: 0.9442
Epoch 10/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1350 - accuracy: 0.9459
Epoch 11/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1433 - accuracy: 0.9573
Epoch 12/20
18/18 [==============================] - 2s 114ms/step - loss: 0.1689 - accuracy: 0.9477
Epoch 13/20
18/18 [==============================] - 2s 114ms/step - loss: 0.1672 - accuracy: 0.9494
Epoch 14/20
18/18 [==============================] - 2s 119ms/step - loss: 0.1960 - accuracy: 0.9477
Epoch 15/20
18/18 [==============================] - 2s 115ms/step - loss: 0.2420 - accuracy: 0.9346
Epoch 16/20
18/18 [==============================] - 2s 120ms/step - loss: 0.1372 - accuracy: 0.9477
Epoch 17/20
18/18 [==============================] - 3s 152ms/step - loss: 0.0994 - accuracy: 0.9669
Epoch 18/20
18/18 [==============================] - 3s 149ms/step - loss: 0.0832 - accuracy: 0.9730
Epoch 19/20
18/18 [==============================] - 3s 172ms/step - loss: 0.0902 - accuracy: 0.9686
Epoch 20/20
18/18 [==============================] - 3s 164ms/step - loss: 0.0658 - accuracy: 0.9817
1/1 [==============================] - 0s 1ms/step - loss: 1.6104 - accuracy: 0.7692
Testing accuracy after 260 additional epochs: 76.9%


Epoch 1/20
18/18 [==============================] - 3s 146ms/step - loss: 0.0918 - accuracy: 0.9730
Epoch 2/20
18/18 [==============================] - 2s 119ms/step - loss: 0.0783 - accuracy: 0.9730
Epoch 3/20
18/18 [==============================] - 2s 115ms/step - loss: 0.0715 - accuracy: 0.9808
Epoch 4/20
18/18 [==============================] - 2s 115ms/step - loss: 0.0661 - accuracy: 0.9791
Epoch 5/20
18/18 [==============================] - 2s 117ms/step - loss: 0.0700 - accuracy: 0.9791
Epoch 6/20
18/18 [==============================] - 2s 116ms/step - loss: 0.0786 - accuracy: 0.9721
Epoch 7/20
18/18 [==============================] - 2s 125ms/step - loss: 0.0601 - accuracy: 0.9765
Epoch 8/20
18/18 [==============================] - 2s 120ms/step - loss: 0.0604 - accuracy: 0.9817
Epoch 9/20
18/18 [==============================] - 2s 116ms/step - loss: 0.0859 - accuracy: 0.9773
Epoch 10/20
18/18 [==============================] - 2s 125ms/step - loss: 0.0954 - accuracy: 0.9712
Epoch 11/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1484 - accuracy: 0.9608
Epoch 12/20
18/18 [==============================] - 2s 118ms/step - loss: 0.2171 - accuracy: 0.9486
Epoch 13/20
18/18 [==============================] - 2s 116ms/step - loss: 0.4613 - accuracy: 0.8736
Epoch 14/20
18/18 [==============================] - 2s 118ms/step - loss: 0.3674 - accuracy: 0.9024
Epoch 15/20
18/18 [==============================] - 2s 117ms/step - loss: 0.3646 - accuracy: 0.9041
Epoch 16/20
18/18 [==============================] - 2s 115ms/step - loss: 0.2455 - accuracy: 0.9198
Epoch 17/20
18/18 [==============================] - 2s 114ms/step - loss: 0.1767 - accuracy: 0.9372
Epoch 18/20
18/18 [==============================] - 2s 122ms/step - loss: 0.1245 - accuracy: 0.9590
Epoch 19/20
18/18 [==============================] - 2s 120ms/step - loss: 0.1304 - accuracy: 0.9520
Epoch 20/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1239 - accuracy: 0.9564
1/1 [==============================] - 0s 2ms/step - loss: 1.7474 - accuracy: 0.7559
Testing accuracy after 280 additional epochs: 75.6%


Epoch 1/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1164 - accuracy: 0.9669
Epoch 2/20
18/18 [==============================] - 2s 115ms/step - loss: 0.0895 - accuracy: 0.9730
Epoch 3/20
18/18 [==============================] - 2s 115ms/step - loss: 0.0999 - accuracy: 0.9669
Epoch 4/20
18/18 [==============================] - 2s 115ms/step - loss: 0.0991 - accuracy: 0.9738
Epoch 5/20
18/18 [==============================] - 2s 117ms/step - loss: 0.1021 - accuracy: 0.9651
Epoch 6/20
18/18 [==============================] - 2s 117ms/step - loss: 0.1009 - accuracy: 0.9677
Epoch 7/20
18/18 [==============================] - 2s 121ms/step - loss: 0.0917 - accuracy: 0.9677
Epoch 8/20
18/18 [==============================] - 2s 126ms/step - loss: 0.0805 - accuracy: 0.9677
Epoch 9/20
18/18 [==============================] - 2s 116ms/step - loss: 0.0667 - accuracy: 0.9765
Epoch 10/20
18/18 [==============================] - 2s 115ms/step - loss: 0.0850 - accuracy: 0.9773
Epoch 11/20
18/18 [==============================] - 2s 116ms/step - loss: 0.0798 - accuracy: 0.9765
Epoch 12/20
18/18 [==============================] - 2s 114ms/step - loss: 0.0738 - accuracy: 0.9765
Epoch 13/20
18/18 [==============================] - 2s 114ms/step - loss: 0.2077 - accuracy: 0.9433
Epoch 14/20
18/18 [==============================] - 2s 114ms/step - loss: 0.1878 - accuracy: 0.9451
Epoch 15/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1562 - accuracy: 0.9529
Epoch 16/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1131 - accuracy: 0.9669
Epoch 17/20
18/18 [==============================] - 2s 115ms/step - loss: 0.1254 - accuracy: 0.9677
Epoch 18/20
18/18 [==============================] - 2s 133ms/step - loss: 0.1290 - accuracy: 0.9564
Epoch 19/20
18/18 [==============================] - 3s 145ms/step - loss: 0.1236 - accuracy: 0.9590
Epoch 20/20
18/18 [==============================] - 3s 140ms/step - loss: 0.0925 - accuracy: 0.9721
1/1 [==============================] - 0s 1ms/step - loss: 1.9642 - accuracy: 0.7559
Testing accuracy after 300 additional epochs: 75.6%



The model trained for 200 additional epochs performed best on held out test data (78.6% accuracy)

# Now we see that there is no longer a clear benefit to further training on this architecture.
# Final best testing result is 78%.

# The input data (including test data) still has several noisy sequences of data that have not been cleaned (e.g. dead signals).
# In other words, there are many sequences in the test data where there are no patterns to pick up, because of noise / chaos in walking data.
# Therefore, 78% test result, for a basic vanilla architecture, seems like a good starting point for further refinement of both data and architecture.

# For further experimental validation, we could cross-validate the test data set with N-folds.
# As well, given this initial result, we could begin to do a more rigorous architectural and hyperparameter search of the LSTM neural network.

# For cleaning data, from visual analysis of it, the following could be explored:
# (a) remove corrupt data, e.g. "dead" periods, where sensor "drifts", reading approximately the same acceleration value (data that is impossible to label)
# (b) remove data where user is not clearly walking forward, e.g. where z_acceleration is not high enough, based on a threshold per user